package jude.java.money.scan;

import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import jude.java.money.beans.Site;
import jude.java.money.beans.WebUrl;
import jude.java.money.service.ScanService;
import jude.java.money.spring.Services;
import jude.java.money.task.SimpleTask;
import jude.java.util.WebUtils;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethodBase;
import org.apache.commons.httpclient.HttpStatus;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.lang.StringUtils;

/**
 * find sites<br/>
 * ignore https sites<br/>
 */
public class SiteScaner extends SimpleTask<String> {
	private static final int MAX_SIZE = 1024 * 1024;		//1 M
	
	private static final int PATTERN_FLAGS = Pattern.CASE_INSENSITIVE | Pattern.MULTILINE | Pattern.DOTALL;
//	private static final String HTTP_URL_PATTERN = "<a\\b[^>]*href=['\\\"](https?://[\\w-/\\\\\\.=?&]{4,100})['\\\"]";
	private static final String URL_PATTERN = "<a\\b[^>]*href=['\"]([\\w./?=&-:;%]{4,200})['\"]";
	
	private static final String DOMAIN_PATTERN = "http://([^\\\\/:?]{4,200})";
	private static final String RESOURCE_PATTERN = "\\.(css|jpg|jpeg|png|js|swf|exe|dll)$";
	
	private HttpClient client;
	private HashSet<String> pending;
	private ScanService service;
	private char[] buffer;
	private int count;
	
	public SiteScaner(String... urls) {
		super("123");
		service = Services.getScanService();
		pending = new HashSet<String>(200);
		buffer = new char[4096];
		count = 0;
		if (urls == null || urls.length == 0) {
			List<WebUrl> us = service.listUnScannedUrls();
			for (WebUrl webUrl : us) {
				pending.add(webUrl.getUrl());
			}
		} else {
			for (int i = 0; i < urls.length; i++) {
				pending.add(urls[i]);
			}
		}
		initHttpClient();
	}
	
	@Override
	protected void execute(String obj) {
		while (!pending.isEmpty()) {
			String url = pending.iterator().next();
			pending.remove(url);
			next(url, true);
			WebUrl wu = service.get(WebUrl.class, "url", url);
			if (wu != null) {
				wu.setStatus(WebUrl.STATE_SCANNED);
				service.update(wu);
			}
		}
	}
	
	private void next(String url, boolean retry) {
		try {
			System.out.println(url);
			GetMethod get = new GetMethod(url);
			WebUtils.pretendRequestHeader(get);
			get.setFollowRedirects(true);
			int statusCode = client.executeMethod(get);
			if (statusCode == HttpStatus.SC_OK) {
				handle(url, get);
//			} else {
//				System.out.println(statusCode + " " + url);
			}
		} catch (org.apache.commons.httpclient.InvalidRedirectLocationException e){
			//give up
		} catch (org.apache.commons.httpclient.CircularRedirectException e) {
			//give up
		} catch (javax.net.ssl.SSLHandshakeException e) {
			//give up
		} catch (org.apache.commons.httpclient.ConnectTimeoutException e) {
			//give up
		} catch (java.net.UnknownHostException e) {
			//give up
		} catch (ThreadDeath e) {
			//give up
		} catch (Throwable e) {
			// other exception, retry
			initHttpClient();
			if (retry) {
				next(url, false);
			} else {
				e.printStackTrace();
			}
		}
	}
	
	private static String normalize(String url) {
		if (StringUtils.isEmpty(url))
			return null;
		try {
			return new URI(url).normalize().toString();
		} catch (URISyntaxException e) {
		}
		return null;
	}
	
	private void handle(String base, HttpMethodBase method) throws IOException {
		StringBuilder sb = new StringBuilder();
		Reader reader = new InputStreamReader(method.getResponseBodyAsStream(), "ISO-8859-1");
		int n;
		while ((n = reader.read(buffer)) != -1) {
			sb.append(buffer, 0, n);
			if (sb.length() >= MAX_SIZE)
				break;
		}
		int start = 0;
		Pattern p = Pattern.compile(URL_PATTERN, PATTERN_FLAGS);
		Matcher m = p.matcher(sb);
		while (m.find(start)) {
			String url = m.group(1);
			start = Math.max(start + 1, m.end(0));
			found(base, url);
		}
	}
	
	private void found(String base, String url) {
		if (StringUtils.isEmpty(url))
			return;
		char c = url.charAt(0);
		if (c == '#')
			return;
		if (url.startsWith("http://")) {// || url.startsWith("https://")
			found(url);
			return;
		}
		
		//ignore like: javascript:void(0), ftp://abc.cn
		//support: /?a=b:c
		int q = url.indexOf('?');
		int u = url.indexOf(':');
		if (q == -1 && u >= 0)
			return;
		if (q >= 0 && u >= 0 && u < q)
			return;
		
//		if (url.indexOf(':') < url.indexOf('?'))	//javascript:void(0)
//			return;
		if (c == '/' || c == '\\') {
			int n = base.indexOf('/', 7);	//"http://".length()
			if (n > 0) {
				url = base.substring(0, n) + url;
			} else {
				url = base + url;
			}
		} else {
			int n = base.indexOf('/', 7);	//"http://".length()
			if (n > 0) {
				n = base.lastIndexOf('/');
				url = base.substring(0, n + 1) + url;
			} else {
				url = base + '/' + url;
			}
		}
		found(url);
	}
	
	private void found(String url) {
//		System.out.println("\t" + url);
		if (StringUtils.isEmpty(url) || url.indexOf("..") >= 0)
			return;
		url = url.replace('\\', '/');
		url = StringUtils.replace(url, "&amp;", "&");
		int n = url.indexOf('#');
		if (n >= 0) {
			url = url.substring(0, n);
		}
		Pattern p = Pattern.compile(RESOURCE_PATTERN, Pattern.CASE_INSENSITIVE);
		Matcher m = p.matcher(url);
		if (m.find())
			return;
		if (pending.contains(url) || !service.save(new WebUrl(url))) {
			return;
		}
		if (pending.size() > 5000) {
			ArrayList<String> us = new ArrayList<String>();
			Iterator<String> it = pending.iterator();
			for (int i = 0; i < 20; i++) {
				us.add(it.next());
			}
			pending.clear();
			pending.addAll(us);
		}
		pending.add(url);
		p = Pattern.compile(DOMAIN_PATTERN, Pattern.CASE_INSENSITIVE);
		m = p.matcher(url);
		if (m.find()) {
			String domain = m.group(1);
			if (service.save(new Site(domain))) {
				count ++;
				System.out.println("\t" + count + " " + domain);
			}
		}
	}
		
	private void initHttpClient() {
		client = new HttpClient();
		client.getHttpConnectionManager().getParams().setConnectionTimeout(20000);
		client.getHttpConnectionManager().getParams().setSoTimeout(60000);
	}
		
	public static String googleSearch(String query) {
		return googleSearch(query, 0);
	}
	
	public static String googleSearch(String query, int page) {
		return googleSearch(query, page, 1)[0];
	}
	public static String[] googleSearch(String query, int start, int count) {
		String[] urls = new String[count];
		StringBuilder sb = new StringBuilder();
		for (int i = 0; i < urls.length; i++) {
			sb.setLength(0);
			try {
				query = URLEncoder.encode(query, "UTF-8");
			} catch (UnsupportedEncodingException e1) {
			}
			sb.append("http://www.google.com/search?");
			if (start == 0 && i == 0) {
				sb.append("hl=en&newwindow=1&q=").append(query).append("&aq=f&aqi=&aql=&oq=&gs_rfai=");
			} else {
				sb.append("q=").append(query).append("&hl=en&newwindow=1&prmd=ni");
//				sb.append("&ei=RT1NTPqLHoryvQOxmvW7Cg");		//unknown
				sb.append("&start=").append((start + i) * 20).append("&sa=N");
			}
			urls[i] = sb.toString();
		}
		return urls;
	}
	
	public static String baiduSearch(String query) {
		return baiduSearch(query, 0);
	}
	
	public static String baiduSearch(String query, int start) {
		return baiduSearch(query, start, 1)[0];
	}
	
	public static String[] baiduSearch(String query, int start, int count) {
//		http://www.baidu.com/s?wd=shoe
		String[] urls = new String[count];
		StringBuilder sb = new StringBuilder();
		for (int i = 0; i < urls.length; i++) {
			sb.setLength(0);
			try {
				query = URLEncoder.encode(query, "UTF-8");
			} catch (UnsupportedEncodingException e1) {
			}
			sb.append("http://www.baidu.com/s?wd=");
			sb.append(query);
			if (start + i > 0) {
				sb.append("&pn=").append(10 * (start + i));
				sb.append("&usm=1");
			}
			urls[i] = sb.toString();
		}
		return urls;
	}
}
